package com.chance.cc.crawler.development.scripts.zhaoshang;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpConstant;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.pipeline.result.CrawlerDataBuilder;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.PathNotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2020/12/22 11:06
 * @Description 招商 产业链招商
 **/
public class ZhaoshangCompanyCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(ZhaoshangCompanyCrawlerScript.class);
    //产业招商链
    private static final String listUrl = "http://zhaoshang.zkdj.com/folder/list";
    private static final String industrialChainCountIndustryNode = "http://zhaoshang.zkdj.com/industrialChain/countIndustryNode";
    private static final String industrialChainSeach = "http://zhaoshang.zkdj.com/industrialChain/serach";

    //主题推荐
    private static final String themeListFindByPages = "http://zhaoshang.zkdj.com/topic/findByPages\\?titleName=&industryId=\\d+&pageSize=\\d+&pageNum=\\d+";
    private static final String themeListSerach = "http://zhaoshang.zkdj.com/accurate/serach";

    //情报资讯
    private static final String newsReportSerach = "http://zhaoshang.zkdj.com/infoprophet/serachList";
    private static final String newsReportEnterpriseInfo = "http://zhaoshang.zkdj.com/infoprophet/getEnterpriseInfo\\?eid=\\S*";

    //中标速递
    private static final String biddingReprotSerach = "http://zhaoshang.zkdj.com/infobid/serachList";
    private static final String biddingReprotEnterpriseInfo = "http://zhaoshang.zkdj.com/infobid/getEnterpriseInfo\\?agencyName=\\S*";

    //通融快报
    private static final String investmentReportList = "http://zhaoshang.zkdj.com/financing/findFinancingList.json";
    private static final String investmentReportEnterpriseInfo = "http://zhaoshang.zkdj.com/financing/getEnterpriseInfo\\?name=\\S*";

    //专利分析招商
    private static final String patentPageSerach = "http://zhaoshang.zkdj.com/zhuanli/search";
    private static final String patentPageCompany = "http://zhaoshang.zkdj.com/zhuanli/get/\\S*";

    //展会活动招商
    private static final String internalExhibitList = "http://zhaoshang.zkdj.com/exhibition/list";
    private static final String internalExhibitRecommend = "http://zhaoshang.zkdj.com/exhibition/recommend";
    private static final String internalExhibitQueryDetails = "http://zhaoshang.zkdj.com/exhibition/queryDetails";
    private static final String internalExhibitHistory = "http://zhaoshang.zkdj.com/exhibition/history";
    private static final String internalExhibitMayExhibitors = "http://zhaoshang.zkdj.com/exhibition/mayExhibitors";

    //企业测评
    private static final String talentRadarList = "http://zhaoshang.zkdj.com/evaluation/findAll";
    private static final String talentRadarEvalutionDetail = "http://zhaoshang.zkdj.com/evaluation/fildAiEvalutionDetail/\\S*";


    //招商智库
    private static final String associationPageList = "http://zhaoshang.zkdj.com/association/pageList";
    private static final String associationPageIndustryEsperList = "http://zhaoshang.zkdj.com/association/industryExpertList\\?expertName=&industryId=&pageSize=\\d+&pageNum=\\d+";
    private static final String associationPageAttentionAtlasList = "http://zhaoshang.zkdj.com/industrialChain/attentionAtlasList";
    private static final String associationPageOrganizationName = "http://zhaoshang.zkdj.com/association/organizationDetail\\?organizationName=\\S*";

    //企业信息
    private static final String companyUrl = "http://zhaoshang.zkdj.com/accurate/findByEid\\?eid=\\S*";
    private static final String getRelationUrl = "http://zhaoshang.zkdj.com/enteratlas/getRelation";
    private static final String getRelatedInfoByEidUrl = "http://zhaoshang.zkdj.com/accurate/getRelatedInfoByEid\\?eid=\\S*";
    private static final String companyInvestment = "http://zhaoshang.zkdj.com/accurate/landCompanyByName\\S*";
    private static final String companyZhuanli = "http://zhaoshang.zkdj.com/zhuanli/zhuanliByCompanyName";
    private static final String companyZhaopin = "http://zhaoshang.zkdj.com/accurate/zhaopinInfoByName\\S*";
    private static final String companyBidInfo = "http://zhaoshang.zkdj.com/accurate/bidInfoByName\\S*";
    private static final String companyFinancingInfo = "http://zhaoshang.zkdj.com/accurate/financingInfoByName\\S*";
    private static final String companyProphetInfo = "http://zhaoshang.zkdj.com/accurate/prophetInfoByEid\\S*";

    private static final String industrialChain = "industrialChain";
    private static final String themeList = "themeList";
    private static final String newsReport = "newsReport";
    private static final String biddingReprot = "biddingReprot";
    private static final String investmentReport = "investmentReport";
    private static final String patentPage = "patentPage";
    private static final String associationPage = "associationPage";
    private static final String internalExhibit = "internalExhibit";
    private static final String talentRadar = "talentRadar";

    /**
     * 脚本domain定义
     *
     * @return
     */
    @Override
    public String domain() {
        return "zhaoshang";
    }

    /**
     * 进入脚本的正则列表
     */
    @Override
    public void initUrlRegulars() {
        addUrlRegular(listUrl);
        addUrlRegular(industrialChainCountIndustryNode);
        addUrlRegular(industrialChainSeach);

        addUrlRegular(themeListFindByPages);
        addUrlRegular(themeListSerach);

        addUrlRegular(newsReportSerach);
        addUrlRegular(newsReportEnterpriseInfo);

        addUrlRegular(biddingReprotSerach);
        addUrlRegular(biddingReprotEnterpriseInfo);

        addUrlRegular(investmentReportList);
        addUrlRegular(investmentReportEnterpriseInfo);

        addUrlRegular(patentPageSerach);
        addUrlRegular(patentPageCompany);

        addUrlRegular(talentRadarList);

        addUrlRegular(internalExhibitList);
        addUrlRegular(internalExhibitRecommend);
        addUrlRegular(internalExhibitHistory);
        addUrlRegular(internalExhibitQueryDetails);
        addUrlRegular(internalExhibitMayExhibitors);

        addUrlRegular(associationPageList);
        addUrlRegular(associationPageIndustryEsperList);
        addUrlRegular(associationPageAttentionAtlasList);
        addUrlRegular(associationPageOrganizationName);

        addUrlRegular(companyUrl);
        addUrlRegular(getRelationUrl);
        addUrlRegular(getRelatedInfoByEidUrl);
        addUrlRegular(companyInvestment);
        addUrlRegular(companyZhuanli);
        addUrlRegular(companyZhaopin);
        addUrlRegular(companyBidInfo);
        addUrlRegular(companyFinancingInfo);
        addUrlRegular(companyProphetInfo);
        addUrlRegular(talentRadarEvalutionDetail);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     *
     * @param crawlerRequestRecord
     * @return
     */
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return true;
    }


    /**
     * 解析链接方法
     *
     * @param crawlerRequestRecord
     * @param httpPage
     * @return
     */
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");

        if (httpPage.isDownloadSuccess() && httpPage.getStatusCode() == 200) {
            String requestUrl = httpPage.getRequest().getUrl();

            if (requestUrl.matches(listUrl)) {
                List<String> all = httpPage.getJson().jsonPath($_type + ".data").all();
                for (String data : all) {
                    JSONObject jsonObject = JSON.parseObject(data);
                    if (industrialChain.equals(site)) {
                        //解析出nodesId的地址
                        String id = jsonObject.getString("id");
                        String countIndestryNode = "http://zhaoshang.zkdj.com/industrialChain/countIndustryNode";
                        CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(countIndestryNode + id)
                                .httpUrl(countIndestryNode)
                                .copyBizTags()
                                .releaseTime(System.currentTimeMillis())
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .build();

                        HttpRequest httpRequest = itemRecord.getHttpRequest();
                        httpRequest.setMethod(HttpConstant.Method.POST);
                        Map<String, Object> map = new HashMap<>();
                        map.put("id", id);
                        httpRequest.setRequestBody(HttpRequestBody.form(map, "UTF-8"));
                        Map<String, String> headers = httpPage.getRequest().getHeaders();
                        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                        httpRequest.setHeaders(headers);
                        httpRequest.setExtras(map);

                        parsedLinks.add(itemRecord);


                        //解析出企业列表
                        String serach = "http://zhaoshang.zkdj.com/industrialChain/serach";
                        CrawlerRequestRecord serachRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(serach + id)
                                .httpUrl(serach)
                                .copyBizTags()
                                .needWashed(true)
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .releaseTime(System.currentTimeMillis())
                                .build();

                        HttpRequest serachHttpRequest = serachRecord.getHttpRequest();
                        serachHttpRequest.setMethod(HttpConstant.Method.POST);
                        Map<String, Object> serachMap = new HashMap<>();
                        serachMap.put("id", id);
                        serachMap.put("nodesId", "");
                        serachMap.put("pageNum", 1);
                        serachMap.put("pageSize", 200);
//                        serachMap.put("pageSize", 3);
                        serachMap.put("order", "cs_sdse");
                        serachMap.put("order_type", "desc");
                        serachHttpRequest.setRequestBody(HttpRequestBody.form(serachMap, "UTF-8"));
                        serachHttpRequest.setHeaders(httpPage.getRequest().getHeaders());
                        serachHttpRequest.setExtras(serachMap);

                        parsedLinks.add(serachRecord);

                    } else if (themeList.equals(site)) {
                        String i_id = jsonObject.getString("i_id");
                        String findByPages = "http://zhaoshang.zkdj.com/topic/findByPages?titleName=&industryId=" + i_id + "&pageSize=200&pageNum=1";
                        CrawlerRequestRecord findByPagesRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(findByPages)
                                .httpUrl(findByPages)
                                .needWashed(true)
                                .copyBizTags()
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .releaseTime(System.currentTimeMillis())
                                .build();

                        HttpRequest httpRequest = findByPagesRecord.getHttpRequest();
                        httpRequest.setHeaders(httpPage.getRequest().getHeaders());
                        Map<String, Object> extras = new HashMap<>();
                        extras.put("i_id", i_id);
                        httpRequest.setExtras(extras);

                        parsedLinks.add(findByPagesRecord);
                    } else if (newsReport.equals(site)) {
                        String i_id = jsonObject.getString("i_id");
                        String serachList = "http://zhaoshang.zkdj.com/infoprophet/serachList";
                        CrawlerRequestRecord serachListRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(serachList + i_id)
                                .httpUrl(serachList)
                                .copyBizTags()
                                .needWashed(true)
                                .releaseTime(System.currentTimeMillis())
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .build();

                        HttpRequest serachListRequest = serachListRecord.getHttpRequest();
                        serachListRequest.setMethod(HttpConstant.Method.POST);
                        Map<String, Object> serachListMap = new HashMap<>();
                        serachListMap.put("mentionState", 1);
                        serachListMap.put("pageNum", 1);
                        serachListMap.put("pageSize", 200);//不管这里填多少，后端已经固定是10
                        serachListMap.put("id", i_id);
                        serachListMap.put("type", 2);
                        serachListRequest.setRequestBody(HttpRequestBody.form(serachListMap, "UTF-8"));
                        Map<String, String> headers = httpPage.getRequest().getHeaders();
                        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                        serachListRequest.setHeaders(headers);
                        serachListRequest.setExtras(serachListMap);

                        parsedLinks.add(serachListRecord);
                    } else if (biddingReprot.equals(site)) {
                        String i_id = jsonObject.getString("i_id");
                        String biddingReprot = "http://zhaoshang.zkdj.com/infobid/serachList";
                        CrawlerRequestRecord biddingReprotRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(biddingReprot + i_id)
                                .httpUrl(biddingReprot)
                                .releaseTime(System.currentTimeMillis())
                                .needWashed(true)
                                .copyBizTags()
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .build();

                        HttpRequest biddingReprotRequest = biddingReprotRecord.getHttpRequest();
                        biddingReprotRequest.setMethod(HttpConstant.Method.POST);
                        Map<String, Object> biddingReprotMap = new HashMap<>();
                        biddingReprotMap.put("pageNum", 1);
                        biddingReprotMap.put("pageSize", 200);
                        biddingReprotMap.put("id", i_id);
                        biddingReprotMap.put("type", 0);
                        biddingReprotRequest.setRequestBody(HttpRequestBody.form(biddingReprotMap, "UTF-8"));
                        Map<String, String> headers = httpPage.getRequest().getHeaders();
                        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                        biddingReprotRequest.setHeaders(headers);
                        biddingReprotRequest.setExtras(biddingReprotMap);

                        parsedLinks.add(biddingReprotRecord);

                    } else if (patentPage.equals(site)) {
                        String i_id = jsonObject.getString("i_id");
                        String patentPageReprot = "http://zhaoshang.zkdj.com/zhuanli/search";
                        CrawlerRequestRecord patentPageRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(patentPageReprot + i_id)
                                .httpUrl(patentPageReprot)
                                .releaseTime(System.currentTimeMillis())
                                .copyBizTags()
                                .needWashed(true)
                                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                                .build();

                        HttpRequest patentPageRequest = patentPageRecord.getHttpRequest();
                        patentPageRequest.setMethod(HttpConstant.Method.POST);
                        Map<String, Object> patentPageMap = new HashMap<>();
                        patentPageMap.put("pageNum", 1);
                        patentPageMap.put("pageSize", 200);
                        patentPageMap.put("industry", i_id);
                        patentPageMap.put("keywordType", "invention_title");
                        patentPageRequest.setRequestBody(HttpRequestBody.form(patentPageMap, "UTF-8"));
                        Map<String, String> headers = httpPage.getRequest().getHeaders();
                        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                        patentPageRequest.setHeaders(headers);
                        patentPageRequest.setExtras(patentPageMap);

                        parsedLinks.add(patentPageRecord);

                    }
                }
            } else if (requestUrl.matches(investmentReportList)) {
                //投融快报翻页
                Map<String, Object> oldMap = httpPage.getRequest().getExtras();
                CrawlerRequestRecord serachListRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(requestUrl + oldMap.get("pageNo"))
                        .httpUrl(requestUrl)
                        .needWashed(true)
                        .copyBizTags()
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .build();

                HttpRequest serachListRequest = serachListRecord.getHttpRequest();
                serachListRequest.setMethod(HttpConstant.Method.POST);
                Map<String, Object> serachListMap = new HashMap<>();
                Set<String> set = oldMap.keySet();
                for (String key : set) {
                    if ("pageNo".equals(key)) {
                        serachListMap.put(key, (Integer) oldMap.get(key) + 1);
                    } else {
                        serachListMap.put(key, oldMap.get(key));
                    }
                }
                serachListRequest.setRequestBody(HttpRequestBody.form(serachListMap, "UTF-8"));
                serachListRequest.setHeaders(httpPage.getRequest().getHeaders());
                serachListRequest.setExtras(serachListMap);

                parsedLinks.add(serachListRecord);
            } else if (requestUrl.matches(themeListFindByPages) || requestUrl.matches(associationPageIndustryEsperList)) {
                //主题findByPages翻页 get
                String[] split = requestUrl.split("pageNum=");
                String nextPageUrl = split[0] + "pageNum=" + (Integer.parseInt(split[1]) + 1);

                CrawlerRequestRecord findByPagesRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .needWashed(true)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .copyBizTags()
                        .releaseTime(System.currentTimeMillis())
                        .build();
                findByPagesRecord.getHttpRequest().setHeaders(httpPage.getRequest().getHeaders());
                findByPagesRecord.getHttpRequest().setExtras(httpPage.getRequest().getExtras());

                parsedLinks.add(findByPagesRecord);
            }

            if (requestUrl.matches(themeListFindByPages)) {
                //theme解析企业列表
                List<String> all = httpPage.getJson().jsonPath($_type + ".data.byPage.rows").all();
                for (String data : all) {
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    String lableId = jsonObject.getString("lableId");
                    String titleName = jsonObject.getString("titleName");
                    String themeCompany = "http://zhaoshang.zkdj.com/accurate/serach";
                    CrawlerRequestRecord themeCompanyRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(themeCompany + lableId)
                            .httpUrl(themeCompany)
                            .needWashed(true)
                            .copyBizTags()
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    HttpRequest httpRequest = themeCompanyRecord.getHttpRequest();
                    httpRequest.setMethod(HttpConstant.Method.POST);
                    Map<String, Object> map = new HashMap<>();
                    map.put("pageNum", 1);
                    map.put("pageSize", 200);
                    map.put("order", "sszb_new");
                    map.put("order_type", "desc");
                    map.put("label", lableId);
                    Map<String, String> heards = new HashMap<>();
                    heards.putAll(httpPage.getRequest().getHeaders());
                    heards.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                    heards.put("Origin", "http://zhaoshang.zkdj.com");
                    heards.put("Referer", "http://zhaoshang.zkdj.com/themeCompany?label=" + lableId + "&titleName=" + titleName + "&flag=1");
                    httpRequest.setRequestBody(HttpRequestBody.form(map, "UTF-8"));
                    httpRequest.setHeaders(heards);
                    httpRequest.setExtras(map);

                    parsedLinks.add(themeCompanyRecord);
                }
            } else if (requestUrl.matches(industrialChainSeach) || requestUrl.matches(themeListSerach) || requestUrl.matches(newsReportSerach) || requestUrl.matches(biddingReprotSerach) || requestUrl.matches(patentPageSerach) || requestUrl.matches(associationPageList)
                    || requestUrl.matches(internalExhibitList) || requestUrl.matches(internalExhibitMayExhibitors) || requestUrl.matches(talentRadarList)) {
                //企业列表翻页 POST
                Map<String, Object> oldMap = httpPage.getRequest().getExtras();
                String recordKey = "";
                if (newsReport.equals(site) || biddingReprot.equals(site) || requestUrl.matches(internalExhibitMayExhibitors)) {
                    recordKey = requestUrl + oldMap.get("id") + oldMap.get("pageNum");
                } else if (industrialChain.equals(site)) {
                    recordKey = requestUrl + oldMap.get("id") + oldMap.get("pageNum");
                } else if (themeList.equals(site)) {
                    recordKey = requestUrl + oldMap.get("label") + oldMap.get("pageNum");
                } else if (patentPage.equals(site)) {
                    recordKey = requestUrl + oldMap.get("industry") + oldMap.get("pageNum");
                } else if (associationPage.equals(site)) {
                    recordKey = requestUrl + oldMap.get("organization") + oldMap.get("pageNum");
                }
                if (requestUrl.matches(internalExhibitList) || talentRadar.equals(site)) {
                    recordKey = requestUrl + oldMap.get("pageNum");
                }
                CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(recordKey)
                        .httpUrl(requestUrl)
                        .needWashed(true)
                        .copyBizTags()
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .releaseTime(System.currentTimeMillis())
                        .build();


                HttpRequest httpRequest = itemRecord.getHttpRequest();
                httpRequest.setMethod(HttpConstant.Method.POST);
                Map<String, Object> map = new HashMap<>();
                Set<String> set = oldMap.keySet();
                for (String key : set) {
                    if ("pageNum".equals(key)) {
                        map.put(key, (Integer) oldMap.get(key) + 1);
                    } else {
                        map.put(key, oldMap.get(key));
                    }
                }
                httpRequest.setRequestBody(HttpRequestBody.form(map, "UTF-8"));
                httpRequest.setHeaders(httpPage.getRequest().getHeaders());
                httpRequest.setExtras(map);

                parsedLinks.add(itemRecord);
            }

            if (requestUrl.matches(associationPageList)) {
                //招商智库pageList详情
                List<String> all = httpPage.getJson().jsonPath($_type + ".data.rows").all();
                for (String data : all) {
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    String organizationName = jsonObject.getString("organizationName");

                    String url = "http://zhaoshang.zkdj.com/association/organizationDetail?organizationName=" + organizationName;
                    CrawlerRequestRecord detailsRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(url)
                            .httpUrl(url)
                            .copyBizTags()
                            .needParsed(false)
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .build();

                    HttpRequest detailsRequest = detailsRecord.getHttpRequest();
                    Map<String, Object> extras = new HashMap<>();
                    extras.put("organizationName", organizationName);
                    detailsRequest.setHeaders(httpPage.getRequest().getHeaders());
                    detailsRequest.addHeader("Referer", "http://zhaoshang.zkdj.com/associationDetails?organizationName=" + organizationName);
                    detailsRequest.setExtras(extras);

                    parsedLinks.add(detailsRecord);
                }
            }

            if (requestUrl.matches(internalExhibitRecommend) || requestUrl.matches(internalExhibitList)) {
                //展会招商活动具体信息
                List<String> all = new LinkedList<>();
                if (requestUrl.matches(internalExhibitRecommend)) {
                    all = httpPage.getJson().jsonPath($_type + ".data").all();
                } else if (requestUrl.matches(internalExhibitList)) {
                    all = httpPage.getJson().jsonPath($_type + ".data.list").all();
                }
                for (String data : all) {
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    String id = jsonObject.getString("id");
                    String overdue = jsonObject.getString("overdue");
                    String name = jsonObject.getString("zhmc");
                    Map<String, Object> extras = new HashMap<>();
                    extras.put("id", id);
                    extras.put("overdue", overdue);
                    extras.put("name", name);

                    Map<String, String> headers = new HashMap<>();
                    headers.putAll(httpPage.getRequest().getHeaders());
                    if (requestUrl.matches(internalExhibitRecommend)) {
                        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                    }
                    headers.put("Referer", "http://zhaoshang.zkdj.com/internalExhibitDetail?id=" + id + "&name=" + name + "&overdue=" + overdue);

                    //展会详情信息
                    String details = "http://zhaoshang.zkdj.com/exhibition/queryDetails";
                    CrawlerRequestRecord detailsRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(details + id)
                            .httpUrl(details)
                            .copyBizTags()
                            .needParsed(false)
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .build();

                    HttpRequest detailsRequest = detailsRecord.getHttpRequest();
                    detailsRequest.setMethod(HttpConstant.Method.POST);
                    Map<String, Object> detailstMap = new HashMap<>();
                    detailstMap.put("id", id);
                    detailsRequest.setRequestBody(HttpRequestBody.form(detailstMap, "UTF-8"));
                    detailsRequest.setHeaders(headers);
                    detailsRequest.setExtras(extras);

                    parsedLinks.add(detailsRecord);


                    //展会历史
                    String history = "http://zhaoshang.zkdj.com/exhibition/history";
                    CrawlerRequestRecord historyRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(history + id)
                            .httpUrl(history)
                            .needParsed(false)
                            .copyBizTags()
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .build();

                    HttpRequest historyRequest = historyRecord.getHttpRequest();
                    historyRequest.setMethod(HttpConstant.Method.POST);
                    Map<String, Object> historyMap = new HashMap<>();
                    historyMap.put("name", name);
                    historyRequest.setRequestBody(HttpRequestBody.form(historyMap, "UTF-8"));
                    historyRequest.setHeaders(headers);
                    historyRequest.setExtras(extras);

                    parsedLinks.add(historyRecord);


                    //可能参展企业
                    String mayExhibitors = "http://zhaoshang.zkdj.com/exhibition/mayExhibitors";
                    CrawlerRequestRecord mayExhibitorsRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(mayExhibitors + id)
                            .httpUrl(mayExhibitors)
                            .needWashed(true)
                            .copyBizTags()
                            .releaseTime(System.currentTimeMillis())
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .build();

                    HttpRequest mayExhibitorsRequest = mayExhibitorsRecord.getHttpRequest();
                    mayExhibitorsRequest.setMethod(HttpConstant.Method.POST);
                    Map<String, Object> mayExhibitorsMap = new HashMap<>();
                    mayExhibitorsMap.put("id", id);
                    mayExhibitorsMap.put("pageSize", 200);
                    mayExhibitorsMap.put("pageNum", 1);
                    mayExhibitorsRequest.setRequestBody(HttpRequestBody.form(mayExhibitorsMap, "UTF-8"));
                    mayExhibitorsRequest.setHeaders(headers);
                    mayExhibitorsRequest.setExtras(extras);

                    parsedLinks.add(mayExhibitorsRecord);
                }
            }

            if (requestUrl.matches(newsReportSerach) || requestUrl.matches(biddingReprotSerach) || requestUrl.matches(investmentReportList)) {
                //情报资讯中相关公司信息
                List<String> all = httpPage.getJson().jsonPath($_type + ".data.rows").all();
                for (String data : all) {
                    String url = "";
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    Map<String, Object> map = new HashMap<>();
                    if (newsReport.equals(site)) {
                        String eid = jsonObject.getString("eid");
                        url = "http://zhaoshang.zkdj.com/infoprophet/getEnterpriseInfo?eid=" + eid;
                        map.put("eid", eid);
                    } else if (biddingReprot.equals(site)) {
                        String agencyName = jsonObject.getString("agencyName");
                        url = "http://zhaoshang.zkdj.com/infobid/getEnterpriseInfo?agencyName=" + agencyName;
                        map.put("agencyName", agencyName);
                    } else if (investmentReport.matches(site)) {
                        String name = jsonObject.getString("financingCompany");
                        url = "http://zhaoshang.zkdj.com/financing/getEnterpriseInfo?name=" + name;
                        map.put("name", name);
                    }

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(url)
                            .httpUrl(url)
                            .needParsed(false)
                            .copyBizTags()
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    itemRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    itemRecord.getHttpRequest().addHeader("Referer", httpPage.getRequest().getHeaders().get("Referer"));
                    itemRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    itemRecord.getHttpRequest().setExtras(map);

                    parsedLinks.add(itemRecord);
                }
            } else if (requestUrl.matches(patentPageSerach)) {
                //专利分析招商详情
                List<String> all = httpPage.getJson().jsonPath($_type + ".data.rows").all();
                for (String data : all) {
                    String id = JSONObject.parseObject(data).getString("id");
                    String url = "http://zhaoshang.zkdj.com/zhuanli/get/" + id;
                    Map<String, Object> map = new HashMap<>();
                    map.put("id", id);

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(url)
                            .httpUrl(url)
                            .needParsed(false)
                            .copyBizTags()
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    itemRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    itemRecord.getHttpRequest().addHeader("Referer", httpPage.getRequest().getHeaders().get("Referer"));
                    itemRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    itemRecord.getHttpRequest().setExtras(map);

                    parsedLinks.add(itemRecord);
                }

            }

            //公司信息
            if (requestUrl.matches(industrialChainSeach) || requestUrl.matches(themeListSerach)) {
                List<String> all = httpPage.getJson().jsonPath($_type + ".data.rows").all();
                for (String data : all) {
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    String eid = jsonObject.getString("id");
                    String companyName = jsonObject.getString("name");


                    //企业测评结果
                    String evalutionDetail = "http://zhaoshang.zkdj.com/evaluation/fildAiEvalutionDetail/" + eid;
                    CrawlerRequestRecord evalutionDetailRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(evalutionDetail)
                            .httpUrl(evalutionDetail)
                            .copyBizTags()
                            .needParsed(false)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    evalutionDetailRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    evalutionDetailRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    evalutionDetailRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    evalutionDetailRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarDetail?eid=" + eid + "&flag=2");
                    Map<String, Object> map = new HashMap<>();
                    map.put("companyId", eid);
                    map.put("companyName", companyName);
                    evalutionDetailRecord.getHttpRequest().setExtras(map);

                    parsedLinks.add(evalutionDetailRecord);

                    //公司详情页
                    String findByEid = "http://zhaoshang.zkdj.com/accurate/findByEid?eid=" + eid;
                    CrawlerRequestRecord findByEidRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(findByEid)
                            .httpUrl(findByEid)
                            .copyBizTags()
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    findByEidRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    findByEidRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    findByEidRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    findByEidRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    Map<String, Object> compayExtras = new HashMap<>();
                    compayExtras.put("companyId", eid);
                    compayExtras.put("companyName", companyName);
                    findByEidRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(findByEidRecord);


                    //对外投资
                    String investment = "http://zhaoshang.zkdj.com/accurate/landCompanyByName?name=" + companyName + "&pageSize=30&pageNum=1";
                    CrawlerRequestRecord investmentRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(investment)
                            .httpUrl(investment)
                            .copyBizTags()
                            .needWashed(true)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    investmentRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    investmentRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    investmentRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    investmentRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    investmentRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(investmentRecord);

//                    专利
                    String zhuanli = "http://zhaoshang.zkdj.com/zhuanli/zhuanliByCompanyName";
                    CrawlerRequestRecord zhuanliRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(zhuanli + companyName)
                            .httpUrl(zhuanli)
                            .copyBizTags()
                            .needWashed(true)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    HttpRequest zhuanliRequest = zhuanliRecord.getHttpRequest();
                    zhuanliRequest.setMethod(HttpConstant.Method.POST);
                    zhuanliRequest.addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    zhuanliRequest.addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    zhuanliRequest.addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    zhuanliRequest.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                    zhuanliRequest.addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    Map<String, Object> zhuanliMap = new HashMap<>();
                    zhuanliMap.put("name", companyName);
                    zhuanliMap.put("pageNum", 1);
                    zhuanliMap.put("pageSize", 30);
                    zhuanliRequest.setRequestBody(HttpRequestBody.form(zhuanliMap, "UTF-8"));
                    zhuanliRequest.setExtras(zhuanliMap);
                    parsedLinks.add(zhuanliRecord);


//                    招聘
                    String zhaopin = "http://zhaoshang.zkdj.com/accurate/zhaopinInfoByName?name=" + companyName + "&pageSize=30&pageNum=1";
                    CrawlerRequestRecord zhaopinRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(zhaopin)
                            .httpUrl(zhaopin)
                            .copyBizTags()
                            .needWashed(true)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    zhaopinRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    zhaopinRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    zhaopinRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    zhaopinRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    zhaopinRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(zhaopinRecord);

//                    投标信息
                    String bidInfo = "http://zhaoshang.zkdj.com/accurate/bidInfoByName?name=" + companyName + "&pageSize=30&pageNum=1";
                    CrawlerRequestRecord bidInfoRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(bidInfo)
                            .httpUrl(bidInfo)
                            .copyBizTags()
                            .needWashed(true)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    bidInfoRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    bidInfoRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    bidInfoRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    bidInfoRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    bidInfoRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(bidInfoRecord);

//                    融资信息
                    String financingInfo = "http://zhaoshang.zkdj.com/accurate/financingInfoByName?name=" + companyName + "&pageSize=30&pageNum=1";
                    CrawlerRequestRecord financingInfoRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(financingInfo)
                            .httpUrl(financingInfo)
                            .copyBizTags()
                            .needWashed(true)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    financingInfoRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    financingInfoRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    financingInfoRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    financingInfoRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    financingInfoRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(financingInfoRecord);

//                    资讯
                    String prophetInfo = "http://zhaoshang.zkdj.com/accurate/prophetInfoByEid?eid=" + eid + "&pageSize=30&pageNum=1";
                    CrawlerRequestRecord prophetInfoRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(prophetInfo)
                            .httpUrl(prophetInfo)
                            .needWashed(true)
                            .copyBizTags()
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                            .releaseTime(System.currentTimeMillis())
                            .build();
                    prophetInfoRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                    prophetInfoRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                    prophetInfoRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                    prophetInfoRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/talentRadarCompany?name=" + companyName + "&id=" + eid);
                    prophetInfoRecord.getHttpRequest().setExtras(compayExtras);

                    parsedLinks.add(prophetInfoRecord);
                }
            }
            //公司中的三部分信息
            else if (requestUrl.matches(companyUrl)) {
                String eid = (String) httpPage.getRequest().getExtras().get("companyId");
                //企业关系图谱
                String getRelation = "http://zhaoshang.zkdj.com/enteratlas/getRelation";
                CrawlerRequestRecord getRelationRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .recordKey(getRelation + eid)
                        .httpUrl(getRelation)
                        .copyBizTags()
                        .needParsed(false)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                HttpRequest getRelationRequest = getRelationRecord.getHttpRequest();
                getRelationRequest.setMethod(HttpConstant.Method.POST);
                Map<String, Object> getRelationMap = new HashMap<>();
                getRelationMap.put("eid", eid);
                Map<String, String> heards = new HashMap<>();
                heards.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                heards.put("Origin", "http://zhaoshang.zkdj.com");
                heards.put("Referer", "http://zhaoshang.zkdj.com/attractInvestment/enterprisetupu.html?eid=" + eid + "&name=undefined");
                heards.put("Host", httpPage.getRequest().getHeaders().get("Host"));
                heards.put("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));

                getRelationRequest.setRequestBody(HttpRequestBody.form(getRelationMap, "UTF-8"));
                getRelationRequest.setHeaders(heards);
                getRelationRequest.setExtras(getRelationMap);

                parsedLinks.add(getRelationRecord);

                //企业投资图谱
                String getRelatedInfoByEid = "http://zhaoshang.zkdj.com/accurate/getRelatedInfoByEid?eid=" + eid + "&type=%E6%8A%95%E8%B5%84%E5%85%B3%E7%B3%BB%E5%9B%BE";
                CrawlerRequestRecord getRelatedInfoByEidRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .recordKey(getRelatedInfoByEid)
                        .httpUrl(getRelatedInfoByEid)
                        .copyBizTags()
                        .needParsed(false)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                getRelatedInfoByEidRecord.getHttpRequest().addHeader("Accept", "application/json, text/javascript, */*; q=0.01");
                getRelatedInfoByEidRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                getRelatedInfoByEidRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                getRelatedInfoByEidRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                getRelatedInfoByEidRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/attractInvestment/Investmenttupu.html?eid=" + eid + "&name=undefined");

                parsedLinks.add(getRelatedInfoByEidRecord);

                //人员图谱
                String peopleRelation = "http://zhaoshang.zkdj.com/accurate/getRelatedInfoByEid?eid=" + eid + "&type=%E4%BC%81%E4%B8%9A%E5%9B%BE%E8%B0%B1";
                CrawlerRequestRecord peopleRelationRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .recordKey(peopleRelation)
                        .httpUrl(peopleRelation)
                        .copyBizTags()
                        .needParsed(false)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                peopleRelationRecord.getHttpRequest().addHeader("Accept", "application/json, text/javascript, */*; q=0.01");
                peopleRelationRecord.getHttpRequest().addHeader("Cookie", httpPage.getRequest().getHeaders().get("Cookie"));
                peopleRelationRecord.getHttpRequest().addHeader("Host", httpPage.getRequest().getHeaders().get("Host"));
                peopleRelationRecord.getHttpRequest().addHeader("User-Agent", httpPage.getRequest().getHeaders().get("User-Agent"));
                peopleRelationRecord.getHttpRequest().addHeader("Referer", "http://zhaoshang.zkdj.com/attractInvestment/Investmenttupu.html?eid=" + eid + "&name=undefined");

                parsedLinks.add(peopleRelationRecord);
            }
            //公司信息中的翻页
            else if (requestUrl.matches(companyInvestment) || requestUrl.matches(companyZhaopin) || requestUrl.matches(companyBidInfo) || requestUrl.matches(companyFinancingInfo) || requestUrl.matches(companyProphetInfo)) {
                String[] split = requestUrl.split("pageNum=");
                String nextPageUrl = split[0] + "pageNum=" + (Integer.parseInt(split[1]) + 1);
                CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .copyBizTags()
                        .needWashed(true)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .releaseTime(System.currentTimeMillis())
                        .build();
                itemRecord.getHttpRequest().setHeaders(httpPage.getRequest().getHeaders());
                itemRecord.getHttpRequest().setExtras(httpPage.getRequest().getExtras());

                parsedLinks.add(itemRecord);
            } else if (requestUrl.matches(companyZhuanli)) {
                Map<String, Object> oldMap = httpPage.getRequest().getExtras();
                CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(requestUrl + oldMap.get("name") + oldMap.get("pageNum"))
                        .httpUrl(requestUrl)
                        .copyBizTags()
                        .needWashed(true)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                HttpRequest httpRequest = itemRecord.getHttpRequest();
                httpRequest.setMethod(HttpConstant.Method.POST);
                Map<String, Object> map = new HashMap<>();
                Set<String> set = oldMap.keySet();
                for (String key : set) {
                    if ("pageNum".equals(key)) {
                        map.put(key, (Integer) oldMap.get(key) + 1);
                    } else {
                        map.put(key, oldMap.get(key));
                    }
                }
                httpRequest.setRequestBody(HttpRequestBody.form(map, "UTF-8"));
                httpRequest.setHeaders(httpPage.getRequest().getHeaders());
                httpRequest.setExtras(map);

                parsedLinks.add(itemRecord);
            }
        } else {
            //下载失败重新推进去进行下载
            String recordKey = crawlerRequestRecord.getRecordKey();
            if (recordKey.matches("\\S*time\\d+")) {
                String[] split = recordKey.split("time");
                int time = Integer.parseInt(split[1]);
                crawlerRequestRecord.setRecordKey(split[0] + "time" + (time + 1));
            } else {
                crawlerRequestRecord.setRecordKey(recordKey + "time1");
            }
            parsedLinks.add(crawlerRequestRecord);
        }
        return parsedLinks;
    }

    @Override
    public void afterInternalDownload(CrawlerRequestRecord crawlerRecord, List<CrawlerRequestRecord> internalDownloadRecords, List<CrawlerRequestRecord> links) {
        Map<String, Object> extras = new HashMap<>();
        for (CrawlerRequestRecord internalDownloadRecord : internalDownloadRecords) {
            HttpPage httpPage = internalDownloadRecord.getInternalDownloadPage();
            String itemUrl = httpPage.getRequest().getUrl();
            List<String> all = new LinkedList<>();
            if (itemUrl.matches(getRelationUrl)) {
                all = httpPage.getJson().jsonPath($_type + ".data.success.results").all();
                extras.put("businessRelations", all);
            } else if (itemUrl.matches(getRelatedInfoByEidUrl)) {
                all = httpPage.getJson().jsonPath($_type + ".data.Node").all();
                if (itemUrl.contains("%E6%8A%95%E8%B5%84%E5%85%B3%E7%B3%BB%E5%9B%BE")) {
                    extras.put("enterpriseInvestment", all);
                } else {
                    extras.put("personnelMap", all);
                }
            }
        }
        crawlerRecord.getHttpRequest().setExtras(extras);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();

        if (crawlerResultTags.hasDataType(CrawlerEnum.CrawlerDataType.article) && page.getJson().jsonPath($_type + ".success").get().equals("true")) {
            if (page.getRequest().getUrl().matches(industrialChainCountIndustryNode)) {
                crawlerDataList.addAll(washListArticleString(crawlerRecord, page));
            } else {
                crawlerDataList.addAll(washListArticle(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }

    public List<CrawlerData> washListArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");

        List<CrawlerData> list = new ArrayList<>();
        List<String> nodes = new ArrayList<>();
        try {

            if (itemUrl.matches(listUrl) || itemUrl.matches(companyUrl) || itemUrl.matches(newsReportEnterpriseInfo) || itemUrl.matches(biddingReprotEnterpriseInfo) || itemUrl.matches(investmentReportEnterpriseInfo)
                    || itemUrl.matches(patentPageCompany) || itemUrl.matches(internalExhibitRecommend) || itemUrl.matches(internalExhibitHistory) || itemUrl.matches(internalExhibitQueryDetails) || itemUrl.matches(talentRadarEvalutionDetail) || itemUrl.matches(associationPageOrganizationName)) {
                nodes = httpPage.getJson().jsonPath($_type + ".data").all();
            } else if (itemUrl.matches(themeListFindByPages)) {
                nodes = httpPage.getJson().jsonPath($_type + ".data.byPage.rows").all();
            } else if (itemUrl.matches(associationPageAttentionAtlasList)) {
                nodes = httpPage.getJson().jsonPath($_type + ".data.investment").all();
            } else if (itemUrl.matches(internalExhibitList)) {
                nodes = httpPage.getJson().jsonPath($_type + ".data.list").all();
            } else {
                nodes = httpPage.getJson().jsonPath($_type + ".data.rows").all();
            }
            for (int i = 0; i < nodes.size(); i++) {
                JSONObject jsonObject = JSONObject.parseObject(nodes.get(i));
                String dataId = "";
                if (itemUrl.matches(companyUrl)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), itemUrl.substring(itemUrl.lastIndexOf("=") + 1), "company");
                } else if (itemUrl.matches(companyBidInfo)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "bidInfo");
                } else if (itemUrl.matches(companyFinancingInfo)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "financingInfo");
                } else if (itemUrl.matches(companyInvestment)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "investment");
                } else if (itemUrl.matches(companyProphetInfo)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "prophetInfo");
                } else if (itemUrl.matches(companyZhaopin)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "zhaopin");
                } else if (itemUrl.matches(companyZhuanli)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "zhuanli");
                } else if (itemUrl.matches(talentRadarEvalutionDetail)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "evalutionDetail");
                } else if (itemUrl.matches(associationPageList)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("organizationName"));
                } else if (itemUrl.matches(associationPageIndustryEsperList)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("dbr"));
                } else if (itemUrl.matches(internalExhibitQueryDetails)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "detail");
                } else if (itemUrl.matches(internalExhibitHistory)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "history");
                } else if (itemUrl.matches(internalExhibitRecommend)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "recommend");
                } else if (itemUrl.matches(internalExhibitList)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "list");
                } else if (itemUrl.matches(patentPageCompany)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "company");
                } else if (itemUrl.matches(newsReportEnterpriseInfo)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"), "enterpriseInfo");
                } else if (itemUrl.matches(associationPageOrganizationName)) {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), itemUrl.substring(itemUrl.lastIndexOf("=") + 1), "organizationDetail");
                } else {
                    dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"));
                }
                CrawlerDataBuilder crawlerDataBuilder = CrawlerData.builder()
                        .data(crawlerRequestRecord, httpPage)
                        .dataId(dataId)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                        .url(itemUrl)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .releaseTime(System.currentTimeMillis());

                if (itemUrl.matches(listUrl) || itemUrl.matches(associationPageList) || itemUrl.matches(internalExhibitList) || itemUrl.matches(talentRadarList)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "List");
                } else if (itemUrl.matches(associationPageOrganizationName)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "OrganizationName");
                } else if (itemUrl.matches(talentRadarEvalutionDetail)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "EvalutionDetail");
                } else if (itemUrl.matches(internalExhibitRecommend)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "Recommend");
                } else if (itemUrl.matches(internalExhibitHistory)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "History");
                } else if (itemUrl.matches(internalExhibitMayExhibitors)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "MayExhibitors");
                } else if (itemUrl.matches(internalExhibitQueryDetails)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "QueryDetails");
                } else if (itemUrl.matches(associationPageIndustryEsperList)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "IndustryEsperList");
                } else if (itemUrl.matches(associationPageAttentionAtlasList)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "AttentionAtlasList");
                } else if (itemUrl.matches(themeListFindByPages)) {
                    crawlerDataBuilder.flowInPipelineTag("themeListFindByPages");
                } else if (itemUrl.matches(industrialChainSeach) || itemUrl.matches(themeListSerach) || itemUrl.matches(newsReportSerach) || itemUrl.matches(biddingReprotSerach) || itemUrl.matches(investmentReportList) || itemUrl.matches(patentPageSerach)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "Seach");
                } else if (itemUrl.matches(newsReportEnterpriseInfo) || itemUrl.matches(biddingReprotEnterpriseInfo) || itemUrl.matches(investmentReportEnterpriseInfo)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "EnterpriseInfo");
                } else if (itemUrl.matches(companyUrl) || itemUrl.matches(patentPageCompany)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "Company");//公司详情页
                } else if (itemUrl.matches(companyInvestment)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyInvestment");
                } else if (itemUrl.matches(companyZhuanli)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyZhuanli");
                } else if (itemUrl.matches(companyZhaopin)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyZhaopin");
                } else if (itemUrl.matches(companyBidInfo)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyBidInfo");
                } else if (itemUrl.matches(companyFinancingInfo)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyFinancingInfo");
                } else if (itemUrl.matches(companyProphetInfo)) {
                    crawlerDataBuilder.flowInPipelineTag(site + "CompanyProphetInfo");
                }

                Set<String> set = jsonObject.getInnerMap().keySet();
                for (String key : set) {
                    crawlerDataBuilder.addContentKV(key, jsonObject.getString(key));
                }

                Map<String, Object> extras = httpPage.getRequest().getExtras();
                if (extras != null) {
                    //添加搜索条件
                    try {
                        if (itemUrl.matches(companyUrl)) {
                            crawlerDataBuilder.addContentKV("company_map", new ObjectMapper().writeValueAsString(extras));
                        } else {
                            crawlerDataBuilder.addContentKV("search_option", new ObjectMapper().writeValueAsString(extras));
                        }
                    } catch (JsonProcessingException e) {
                        log.warn("map转换json字符串发生错误！");
                    }
                }

                CrawlerData crawlerData = crawlerDataBuilder.build();
                crawlerData.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
                crawlerData.setFilterPipelineResult(true);
                list.add(crawlerData);
            }
        } catch (PathNotFoundException e) {
            log.warn("该公司(" + httpPage.getRequest().getUrl() + ")没有详细信息");
        }

        return list;
    }

    public List<CrawlerData> washListArticleString(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();

        List<CrawlerData> list = new ArrayList<>();
        JSONArray jsonArray = new JSONArray();
        if (itemUrl.matches(industrialChainCountIndustryNode)) {
            jsonArray = JSONObject.parseObject(httpPage.getJson().jsonPath($_type + ".data").get()).getJSONArray("nodes");
        }
        for (int i = 0; i < jsonArray.size(); i++) {
            JSONObject jsonObject = jsonArray.getJSONObject(i);
            String dataId = StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), jsonObject.getString("id"));
            CrawlerDataBuilder crawlerDataBuilder = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(dataId)
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(itemUrl)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .releaseTime(System.currentTimeMillis());

            if (itemUrl.matches(industrialChainCountIndustryNode)) {
                crawlerDataBuilder.flowInPipelineTag("industrialChainCountIndustryNode");
            }

            Set<String> set = jsonObject.getInnerMap().keySet();
            for (String key : set) {
                crawlerDataBuilder.addContentKV(key, jsonObject.getString(key));
            }

            Map<String, Object> extras = httpPage.getRequest().getExtras();
            if (extras != null) {
                //添加搜索条件
                try {
                    crawlerDataBuilder.addContentKV("search_option", new ObjectMapper().writeValueAsString(extras));
                } catch (JsonProcessingException e) {
                    log.warn("map转换json字符串发生错误！");
                }
            }

            CrawlerData crawlerData = crawlerDataBuilder.build();
            crawlerData.setFilter(CrawlerEnum.CrawlerRecordFilter.key);
            crawlerData.setFilterPipelineResult(true);
            list.add(crawlerData);
        }

        return list;
    }

    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

}
